{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "e5ksBr8d6uhy"
   },
   "source": [
    "# 3.12 权重衰减\n",
    "\n",
    "## 3.12.1 方法\n",
    "\n",
    "## 3.12.2 高维线性回归实验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "from tensorflow.keras import layers, models, initializers, optimizers\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_train, n_test, num_inputs = 20, 100, 200\n",
    "true_w, true_b = tf.ones((num_inputs, 1)) * 0.01, 0.05\n",
    "\n",
    "features = tf.random.normal(shape=(n_train + n_test, num_inputs))\n",
    "labels = tf.keras.backend.dot(features, true_w) + true_b\n",
    "labels += tf.random.normal(mean=0.01, shape=labels.shape)\n",
    "train_features, test_features = features[:n_train, :], features[n_train:, :]\n",
    "train_labels, test_labels = labels[:n_train], labels[n_train:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lmtuMj-XZOMG"
   },
   "source": [
    "## 3.12.3 从零开始实现\n",
    "\n",
    "### 3.12.3.1 初始化模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_params():\n",
    "    w = tf.Variable(tf.random.normal(mean=1, shape=(num_inputs, 1)))\n",
    "    b = tf.Variable(tf.zeros(shape=(1,)))\n",
    "    return [w, b]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gnOCGqteZ-no"
   },
   "source": [
    "### 3.12.3.2 定义L2范数惩罚项"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def l2_penalty(w):\n",
    "    return tf.reduce_sum((w**2)) / 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "r5MT50ODbNhh"
   },
   "source": [
    "### 3.12.3.3 定义训练和测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linreg(X, w, b):\n",
    "    return tf.matmul(X, w) + b\n",
    "\n",
    "def sgd(params, lr, batch_size, grads):\n",
    "    \"\"\"Mini-batch stochastic gradient descent.\"\"\"\n",
    "    for i, param in enumerate(params):\n",
    "        param.assign_sub(lr * grads[i] / batch_size)\n",
    "\n",
    "def set_figsize(figsize=(3.5, 2.5)):\n",
    "    \"\"\"Set matplotlib figure size.\"\"\"\n",
    "    plt.rcParams['figure.figsize'] = figsize\n",
    "\n",
    "def semilogy(x_vals, y_vals, x_label, y_label, x2_vals=None, y2_vals=None,\n",
    "             legend=None, figsize=(3.5, 2.5)):\n",
    "    \"\"\"Plot x and log(y).\"\"\"\n",
    "    set_figsize(figsize)\n",
    "    plt.xlabel(x_label)\n",
    "    plt.ylabel(y_label)\n",
    "    plt.semilogy(x_vals, y_vals)\n",
    "    if x2_vals and y2_vals:\n",
    "        plt.semilogy(x2_vals, y2_vals, linestyle=':')\n",
    "        plt.legend(legend)\n",
    "    plt.show()\n",
    "    \n",
    "def squared_loss(y_hat, y): \n",
    "    # 注意这里返回的是向量, 另外, pytorch里的MSELoss并没有除以 2\n",
    "    return (y_hat - tf.reshape(y, y_hat.shape)) ** 2 / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size, num_epochs, lr = 1, 100, 0.003\n",
    "net, loss = linreg, squared_loss\n",
    "optimizer = tf.keras.optimizers.SGD()\n",
    "train_iter = tf.data.Dataset.from_tensor_slices(\n",
    "    (train_features, train_labels)).batch(batch_size).shuffle(batch_size)\n",
    "\n",
    "def fit_and_plot(lambd):\n",
    "    w, b = init_params()\n",
    "    train_ls, test_ls = [], []\n",
    "    for _ in range(num_epochs):\n",
    "        for X, y in train_iter:\n",
    "            with tf.GradientTape(persistent=True) as tape:\n",
    "                # 添加了L2范数惩罚项\n",
    "                l = loss(net(X, w, b), y) + lambd * l2_penalty(w)\n",
    "            grads = tape.gradient(l, [w, b])\n",
    "            sgd([w, b], lr, batch_size, grads)\n",
    "        train_ls.append(tf.reduce_mean(loss(net(train_features, w, b),\n",
    "                             train_labels)).numpy())\n",
    "        test_ls.append(tf.reduce_mean(loss(net(test_features, w, b),\n",
    "                            test_labels)).numpy())\n",
    "    semilogy(range(1, num_epochs + 1), train_ls, 'epochs', 'loss',\n",
    "                 range(1, num_epochs + 1), test_ls, ['train', 'test'])\n",
    "    print('L2 norm of w:', tf.norm(w).numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sj0gZgkltES_"
   },
   "source": [
    "### 3.12.3.4 观察过拟合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAC1CAYAAAC56YFuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAfPklEQVR4nO3deXxU9bn48c+TZJLJQkJIAAmBhH2THRHEWlDZrbXVul2stV67l9pbreBSf/ZeFdt6q/a6K9rWrRata1hEQLFsJggSlpAAAZIQAoHse+b7++NMNkhCEmbmTJLn/XrllTxn5pzzzOTkyTlnvuc8YoxBKaUAAuxOQCnlP7QgKKXqaUFQStXTgqCUqqcFQSlVTwuCUqpekN0J+EJsbKxJTEy0Ow2lbJeSknLSGNO7pce7RUFITEwkOTnZ7jSUsp2IHG7tcT1kUErV04KglKqnBUEpVU8LQp0P74S9H1o/GwP/fgqOfmnFtTWw+RnITrHimkorPrbTiqvKrDh3lxVXFsPmp+H4bisuL7DivH1WXHbKik+mW3HpSSvOP2DFxcet+HSmFRflWHHBESsuOGrFRTlWfDrTiotzrfjUQSsuOWHFJzOsuOyUFZ9Is+LyAis+vseKK4utOHeXFVeVWvGxnVZcXWHF2SlWXFttxVnJVuxyWfGRrdb7UefwJtjyXEN8aCNse7EhPrgBvny5IT6wDlJebYjT18L2vzfE+1fDjjca4n1JsPOthnjvh/D1PxviPe9D6jsNceo7sPu9hnjXiobfPcDOf8C+jxviHW9A2qqGePvfIf2ThjjlVcj4tCFOXm69pjrbXoRDnzfEW5+33pM6m5+BI1usn10u671ssu09fca29zTk7LDiqjIrrtv2zpMWBABXrbUR1v1Bumrhkwfg4Hp3XA2rlzb8UqvLrTjz31ZcVWrFdb/UiiJYfa/1hwJQfsqKj7l/iaUnrDj3aysuPmbFeXusuDDLik/st+LTh624Lr9TB6y4rmCcTLfiwiwrzttrxcXugpH7tRWX5Flxzg4rLsu34uxkK64otOKjW624riAc3mTFNeVWfGijFdcVhAPrrdi4C0LGJ7Dmvob3d/8qWPtgQ5yWBJ/+viHe8wGsf6QhTn0XNjzWEO96Gzb+qSHe+SZ88eeGeMfrsOkvDXHKX2FLo4L05cuw9YWGeNuLkNyoAG15pmkB2vwX+Oq1hviLJ6x11tn4OHz9dkP82R9g97sN8fpHrCJUZ91/Ny0wnzwIaSsb4jX3Nyowxnovm2x79zZsezUVVny48bZ3b8O2d56kO1ztOGXKFNOuTxmMsd7oQAcEhVhxRSEEOcHhPDt2uaCyCByh1vNdLqgqhqBQCAq2CkxVybljR5i1ztoaqC5tJg6HwKBm4mqoLjs7Do6AgECoqbL+mM8Z94CAAOu/UE3F2XFIJIicHVdXQG3l2bEzyno/q8uhtuoccTU4I624qgxcNU1jUwshPVqIS61i1CQ2EBJhxZUl1vfWYhEIDnfHxSABZ8SBEBzWfFxRBAFBTeNAh7U9gLWtBAafIw5p2LYqi86Og5wN22LjuH5bc8fnICIpxpgpLT6uBUGp7uNcBUEPGQCXy5BfUml3GkrZTgsCcMvyrSx+6yu701DKdloQgMuG9ebfGfmkZhfanYpStuqUBUFEBovIyyKywhPLu+nigUSEBPH85wc9sTilOi2/KQgislxE8kQk9Yzp80QkTUQyRGQJgDHmoDHmdk+tO9Lp4OaLB5K06xhHT5V5arFKdTp+UxCAV4F5jSeISCDwNDAfGA3cJCKjvbHy22YkEiDwgu4lqG7MbwqCMeZz4NQZk6cCGe49girgLeDb3lh/v6hQrps8gH98eZTsgnJvrEIpv+c3BaEF/YGjjeIsoL+IxIjIc8BEEVna3Iwi8iMRSRaR5BMnTrRpZb+4fCgA/7cu4/yyVqqT8veCIM1MM8aYfGPMT4wxQ4wxjzY3ozHmBWPMFGPMlN69W7wfRBP9e4Zy49QB/DP5KEfy9VyC6n78vSBkAQMaxfFAjjdX+PNZQwkMEP68dr83V6OUX/L3gvAlMExEBolIMHAj8IE3V9g30sltMwbx3o5s9uQUeXNVSvkdvykIIvImsBkYISJZInK7MaYG+AWwGtgLvG2M2e3tXH76zSH0CAniD6v3eXtVSvkVv7mnojHmphamJwFJvswlKszBz2cN5dGV+9h04CSXDIn15eqVso3f7CH4m1svSSQuysmjSftwubr+FaFKgRaEFjkdgdw1dwS7sgv58GuvnsdUym9oQWjFNRP6M7pfJH9YlUZFda3d6SjldVoQWhEQINy3cBTZBeX8bXOm3eko5XVaEM5hxtBYZo3ozV/WZXC6tMrudJTyKi0IbbB0wShKK2t48tN0u1NRyqu0ILTB8L49uHHqQF7bcphDJ0vtTkcpr9GC0Ea/vnI4IUEBLFu51+5UlPIaLQht1LtHCD+dOYTVu4+z9WC+3eko5RVaENrh9ksH0y/KycNJe3WwkuqStCC0Q2hwIHfPHcHXWYV8sFMHK6muRwtCO10zoT9j+0fxh1X7dLCS6nK0ILRT3WClnMIKXv7ikN3pKOVRWhA6YNrgGGaP7ssz6zM4Uawdn1TXoQWhg5bOH0lljUvvrKS6lE5ZEETkGhF5UUTeF5E5duQwuHcEi6Yl8Na2I6TlFtuRglIe5/OC0J6GLC0xxrxnjLkD+AFwgxfTbdWvrhhGREgQjyTpYCXVNdixh/AqbWzIIiJjReSjM776NJr1fvd8togOD2bxFcP4bP8JPtvftlu9K+XPfF4Q2tOQxRizyxhz1RlfeWJ5DFhpjNnu69fQ2C3TExjYK4xHPt5LTa3LzlSUOm/+cg6h2YYsrTz/l8CVwHUi8pPmntCRRi0dERIUyJL5I0k7Xsw/U7K8th6lfMFfCkKzDVlaerIx5iljzGR3s5bnWnhOuxu1dNT8Cy9gSkI0j6/ZT0lljVfXpZQ3+UtB8HlDFk8SEe6/ajQnSyp5bsMBu9NRqsP8pSD4vCGLp00Y0JOrx8fx4saD5GizWNVJ2fGxo980ZPG0384bgQH+uDrN7lSU6hCfN2rxp4YsnhYfHcbtlw7i2Q0HuG1GIuPie9qdklLt4i+HDF3Gz2YOISY8mP/5eC/G6D0TVOeiBcHDejgd3Dl7ONsOnWL17uN2p6NUu2hB8IKbLhrA0D4RLFu5l6oaHaykOg8tCF4QFBjAvQtGkplfxmtbDtudjlJt5jfdn7uaWSP6cOnQWJ5al861k+KJCnPYnZICqqurycrKoqKiwu5UvMrpdBIfH4/D0b7tTguCl4gI9y4YxcK/bOQv69K5/6rRdqekgKysLHr06EFiYiIizQ2Q7fyMMeTn55OVlcWgQYPaNa8eMnjR6LhIrp88gL9uzuRwvjZ48QcVFRXExMR02WIA1j+jmJiYDu0FaUHwst/MGY4jMIDHVu2zOxXl1pWLQZ2OvkYtCF7WJ9LJjy8bQtKuXJIzz7zqWyn/ogXBB+64bBB9I0N0sJKioKCAZ555pt3zLViwgIKCAi9k1JQWBB8ICw7irjkj2HG0gA+/PmZ3OspGLRWE2trWe3wkJSXRs6f3h8JrQfCR706KZ3S/SB5bqQ1eurMlS5Zw4MABJkyYwEUXXcSsWbO4+eabGTt2LADXXHMNkydPZsyYMbzwwgv18yUmJnLy5EkyMzMZNWoUd9xxB2PGjGHOnDmUl3vu6lr92NFHAt0NXv7jpa28uimTn3xziN0pdXsPfbibPTlFHl3m6LhIHvzWmBYfX7ZsGampqezYsYMNGzawcOFCUlNT6z8eXL58Ob169aK8vJyLLrqIa6+9lpiYmCbLSE9P58033+TFF1/k+uuv55133mHRokUeyV/3EHxoxtBYLh/Zh6fXZZBfog1eFEydOrXJWIGnnnqK8ePHM23aNI4ePUp6evpZ8wwaNIgJEyYAMHnyZDIzMz2WT5v2EETkV8ArQDHwEjARWGKMWeOxTLqJexeMZO4TG3nq03Qe+vaFdqfTrbX2n9xXwsPD63/esGEDa9euZfPmzYSFhTFz5sxmxxKEhITU/xwYGOjRQ4a27iH80BhTBMwBegO3Acs8lkU3MrRPD26aOoDXth7hwIkSu9NRPtajRw+Ki5tv7FNYWEh0dDRhYWHs27ePLVu2+Di7theEulEOC4BXjDE7af7GqD4hIjNFZKOIPCciM+3Ko6PuvHI4oY5AHk3SwUrdTUxMDDNmzODCCy/k7rvvbvLYvHnzqKmpYdy4cTzwwANMmzbN5/m19aRiioisAQYBS0WkB9Ch63pFZDlwFZBnjLmw0fR5wJNAIPCSMaa1PRADlABOrBu0diqxESH8bNYQ/rAqjc0H8pk+JObcM6ku44033mh2ekhICCtXrmz2sbrzBLGxsaSmNjQ9u+uuuzyaW1v3EG4HlgAXGWPKAAfWYUNHvMr5d27aaIyZD9wDPNTBPGz1wxmD6N8zlIeT9uBy6WAl5R/aWhCmA2nGmAIRWYTVQq2wIyv0ROcmY0zd3slpIIROyOkI5O65I0jNLuK9Hdl2p6MU0PaC8CxQJiLjgd8Ch4G/eTCPdnVuEpHvisjzwN+B/2vhOT7p3HQ+rh4fx7j4KP64Oo3yKh2spOzX1oJQY6xB+N8GnjTGPAn08GAe7e3c9K4x5sfGmBuMMRtaeI7POjd1VECAcN+CURwrrGD5vw/ZnY5SbS4IxSKyFLgF+Nh9zO/JWwB16s5N5+PiwTHMGd2XZ9ZncKJYByspe7W1INwAVGKNR8jF2p3/owfz6PSdm87Hkvkjqaxx8ee1++1ORXVzbSoI7iLwOhAlIlcBFcaYDp1D6MqdmzpqcO8IFk1L4K1tR9h/vPlBK6pr6OjlzwBPPPEEZWVlHs6oqTYVBBG5HtgGfA+4HtgqItd1ZIXGmJuMMf2MMQ5jTLwx5mX39CRjzHBjzBBjzMMdWXZntviKYYSHBPFI0l67U1Fe5O8Foa0Dk+7DGoOQByAivYG1wApvJdbd9AoP5hezhvLoyn1sTD/BN4b554lQdX4aX/48e/Zs+vTpw9tvv01lZSXf+c53eOihhygtLeX6668nKyuL2tpaHnjgAY4fP05OTg6zZs0iNjaW9evXeyW/tp5DCKgrBm757ZhXtdGtlyQSHx3Kwx/vpVYHK/nGKwvhq9etn2urrXjnP6y4qsyKU9+x4opCK97jPr1Vmm/Fae7RhcXn7tS1bNkyhgwZwo4dO5g9ezbp6els27aNHTt2kJKSwueff86qVauIi4tj586dpKamMm/ePBYvXkxcXBzr16/3WjGAtv9RrxKR1SLyAxH5AfAxnbwxqz9yOgL57byR7Mst5p3tnW5EtmqnNWvWsGbNGiZOnMikSZPYt28f6enpjB07lrVr13LPPfewceNGoqKifJZTmw4ZjDF3i8i1wAysMQMvGGP+5dXMuqlvjevH8i8O8fiaNK4a14+wYL2HjVfd9nHDz4GOpnFwWNPYGdU0Do9pGvfo265VG2NYunQpP/7xj896LCUlhaSkJJYuXcqcOXP43e9+165ld1Sbd/uNMe8YY/7LGPNrLQbeIyLcv3AUx4sqefFzHazU1TS+/Hnu3LksX76ckhLrMvjs7Gzy8vLIyckhLCyMRYsWcdddd7F9+/az5vWWVv/9iEgxzY8YFMAYYyK9klU3NyWxF/MvvIDnPz/ATVMH0CfSaXdKykMaX/48f/58br75ZqZPnw5AREQEr732GhkZGdx9990EBATgcDh49tlnAfjRj37E/Pnz6devn9fOI0h3uC34lClTTHJyst1ptEvmyVJm//kzrp0Uz7Jrx9mdTpexd+9eRo0aZXcaPtHcaxWRFGPMlJbm0U8K/FRibDi3TEvk7eSj7Mv17I1AlWqJFgQ/tviKofRwOnhYG7woH9GC4Md6hgXzy8uHsjH9JBv2++cl3J1RdyiuHX2NWhD83PenJ5IQE8ajSXupqe3QXetUI06nk/z8/C5dFOrawTud7T8ZrR9y+7ngoADumTeSn72+nRUpWdw4daDdKXVq8fHxZGVl4a83zfEUp9NJfHx8u+fTgtAJzL/wAiYN7Mnjn+znW+PjCA/RX1tHORyOJo1RVFN6yNAJiAj3LRzNieJKnv/8oN3pqC5MC0InMTkhmoXj+vHC5wfILTy7m49SntDpCoKIDBSRD0RkuYgssTsfX1oybyQuFzy+Js3uVFQX5dOC4P4jzhOR1DOmzxORNBHJaMMf+XDgY2PMD7F6OHQbA3qF8YMZiazYnuXxrsVKge/3EF7l/Ju0fAXcKCLrAO9dGO6nfj5zKFGhDh5J0sFKyvN8WhA80aQFq2PUg8aYy4GFvszfH0SFOVh8+TC+yNDBSsrz/OEcQruatACrgMUi8hyQ2dKTOkOjlo5aNC2BxJgwHvlYByspz/KHgtDeJi2pxpjrjDE/Mca02OmyMzRq6ajgoACWzB9Fel4Jb247Ync6qgvxh4LQbZu0nI+5Y/oybXAv/veT/RSWV9udjuoi/KEgdOsmLR0lIjxw1WgKyqt5Zn2G3emoLsLXHztqkxYPGhMXxXcm9ueVTZlkF5TbnY7qAnz9KYM2afGw38wZAehgJeUZ/nDIoM5D/56h3HZJIv/6KpvdOYV2p6M6OS0IXcDPZg2lZ6jeWUmdPy0IXUBUqIM7rxzOpgP5fLo379wzKNUCLQhdxM0XD2Rw73Ae0TsrqfOgBaGLcAQGsGTeSA6eLOXdr7LtTkd1UloQupDZo/syLj6Kpz5Np6pG9xJU+2lB6EJEhP+aPZys0+W8nXz03DModQYtCF3MN4f3ZkpCNE+sTae4Qoc0q/bRgtDF1A1pzi+t5Mm16XanozoZLQhd0PgBPbnxogG8simT/ce92y1YdS1aELqou+eOJCIkiP/+aI/dqahORAtCF9UrvKEN3BfpJ+1OR3USWhC6sFumJxAfHcqjK/ficumQZnVuWhC6sJCgQO6aM4LdOUW8vvWw3emoTkALQhd39fg4vjm8N7//aA9fHTltdzrKz2lB6OICAoQnb5xA30gnP31tu95IRbXK7wuCiAwWkZdFZEVr01TLeoYF8/wtkymtrOGG5zdz9FSZ3SkpP+XVguCJTk3ufg23n2uaat2YuChev+NiisqrueH5zWTkldidkvJD3t5DeJXz79SkPGRcfE/e/NE0qmpdfO+5TWzXcwrqDF4tCB7q1KQ8aExcFCt+cgkRziCue3YTD76fSkFZld1pKT9hxzmEdnVqEpEYd5emiSKytKVpzczXZTs3na/E2HA++uU3uGVaAn/fcpgZy9bx8Md79ISjQrx9Dz4RSQQ+MsZc6I6/B8w1xvynO74FmGqM+aW3cpgyZYpJTk721uI7tbTcYp7dkMGHXx/DGMPlI/syKaEnM4bEMn5AT7vTUx4mIinGmCktPR7ky2TctFOTHxlxQQ+euHEid80dwWtbjvDhzhzW7j1OgKSx7NpxXD9lwLkXoroMOwpCfacmIBurU9PNNuShGomPDmPJ/JEsmT+S06VVLH7rK3674mu2Hz7N9CExGAMnSyoZExfFpISehAQFtrgsl8tQUlVDpNPhw1egPMGrBcHdqWkmECsiWVht3F8WkbpOTYHAcu3U5F+iw4N56dYpPPBeKu/vyOGtL5vefSk4MID+0aFc2D+KW6YlMC4+isLyaqJCHZwuq+LOt3aw9dApLhvem2sn9WfkBZEM6R1OUKDfD3vp9rx+DsEf6DmEjquudZF+vITgoAB6hjn46kgBKYdPc+RUKf/OyG/SaFYEHAEBBAUK106KZ/XuXPKKKwGrocwPLkkkKszB6dIqRvWLZFJCNBEhduykdl/nOoegBUF1WFlVDR/tPMaJkkoiQx2cKqnidFkV35+ewODeEVTXuth/vJi9x4p5+8ujbMts+gl0REgQf7xuHJcMjeWNrUeornUxNj6Kiwf1Iiy4+UJRXevicH4pfSKdfnlIYoyh1mX8dm9ICwJaEPzFgRMlBAUIkU4Hu7ILefyT/ew8WkBESBAllTWIgDHgdARwUWIvSitrKCyvJkCk/rEjp8qodN9RenDvcH51xTCuHh+HiJy1vqoaF2m5xQzrG4HT0fI5D09wuQyrdufyx9VpHC+qYOaI3gzoFUZVjYuoUAcXRDoZFBtOVa2Ld7dnY4zhh5cOYly8bz/J0YKAFgR/VVlTy2Mr08gpKOcXlw8lISaMnUcL+WRPLl9mnqZnmIPosGAMBpf7rvLx0aGM6hdJblEFK1OPkZpdxIBeoQSKUGsMjsAAxvaPYtLAaF7dlMmhk6UEBwbwjWGx3LtwFEN6R3gkd5fLkFtUwa7sQrYePMWq1GPkFFYwvG8EkwZGs25fHgXl1QQHBlBSWdNk3khnEAYorqghKtRBVKiDgb3CGHlBD741Po5x8VGICMYYMvPL6BUeTFSoZ/aGtCCgBaGrqnUZ3k4+yoa0PJyOQAJEKK+qZeuhfE6XVTO0TwT/eekgMvJK+EfyUSqrXYyLjyKvuJLAACE2Ipgl80cyOaFXq+t4Y+thVmzPpqi8morqWoyBU2VV9b0vggMDuGx4LFdP6M/Csf0IDGi6t1JV4+J4UQUHT5ZSWV3LZcN7U13rYkVKFodOllJQVk1mfilpucVU1riIi3ISExHCscIKTpZUEiAwcWA035+ewFXj4th/vJj9x4sREXqFBTOkTzj7j5ewfl8et81IJCEmvMXXowUBLQjdTU2ti/S8Eob2icDhPpbPK67g8dX7OZRfygWRTlzG8NWRAvJLK7l3wSi2Hz5Nak5Rk2a5EU4HJRXVHDhRyrj4KBJiwnEGBSBiXUE6sFcYo+MiGd0v0iOHJEUV1Xy4M4dth05RWF5Nz1AHUwfFkFtYTlJqLhl5JYQFB1JWVdvs/CFBATx54wTmXdivxXVoQUALgmreyZJKbl2+jd05RfRwBjF9cEx9ATEYiitqqKxx8f3pCSwc26/Z8xS+4nIZVu/OZX1aHpMTopk4MJoAEfKKKsg4UcKA6DCmDY4hNLj1wqQFAS0IqmUllTVsOZDPjKGx5/xj6gr8ceiyUn4jIiSIK0f3tTsNv+GfH5YqpWyhBUEpVU8LglKqnhYEpVS9bvEpg4icAFrqVBIL+FOvM82ndZpP686VT4IxpndLD3aLgtAaEUlu7WMYX9N8Wqf5tO5889FDBqVUPS0ISql6WhDgBbsTOIPm0zrNp3XnlU+3P4eglGqgewhKqXrduiC0p8ekF9Y9QETWi8heEdktIr9yT+8lIp+ISLr7e7SP8woUka9E5CO78xGRniKyQkT2ud+n6Tbn82v37ypVRN4UEacv82muV2pr6xeRpe5tO01E5rZlHd22ILTUY9KHKdQAvzHGjAKmAT93r38J8KkxZhjwqTv2pV8BexvFdubzJLDKGDMSGO/Oy5Z8RKQ/sBiY4m46FIjVQsCX+bzKGb1SW1q/e1u6ERjjnucZ9zbfOmNMt/wCpgOrG8VLgaU25vM+MBtIA/q5p/UD0nyYQ7x7o7ocq9sWduUDRAKHcJ/najTdrnzqWhD2wrpK+CNgjq/zARKB1HO9H2duz1htD6afa/nddg+BdvaY9CZ3u7uJwFagrzHmGID7uy87YD8B/BZwNZpmVz6DgRPAK+5DmJdEJNyufIwx2cCfgCPAMaDQGLPGrnwaaWn9Hdq+u3NBaO72Nz7/yEVEIoB3gDuNMUW+Xn+jPK4C8owxKXblcIYgYBLwrDFmIlCK7w+f6rmPzb8NDALigHARWWRXPm3Qoe27OxcE23tMiogDqxi8box51z35uIj0cz/eD8jzUTozgKtFJBN4C7hcRF6zMZ8sIMsYs9Udr8AqEHblcyVwyBhzwhhTDbwLXGJjPnVaWn+Htu/uXBDqe0yKSDDWCZgPfLVysW7Q9zKw1xjzv40e+gC41f3zrVjnFrzOGLPUGBNvjEnEei/WGWMW2ZhPLnBUREa4J10B7LErH6xDhWkiEub+3V2BdZLTrnzqtLT+D4AbRSTE3Ud1GLDtnEvzxQkZf/0CFgD7gQPAfT5e96VYu3BfAzvcXwuAGKwTe+nu771seF9m0nBS0bZ8gAlAsvs9eg+Itjmfh4B9QCrwdyDEl/kAb2Kdv6jG2gO4vbX1A/e5t+00YH5b1qEjFZVS9brzIYNS6gxaEJRS9bQgKKXqaUFQStXTgqCUqqcFQdlORGbWXV2p7KUFQSlVTwuCajMRWSQi20Rkh4g87753QomIPC4i20XkUxHp7X7uBBHZIiJfi8i/6q7TF5GhIrJWRHa65xniXnxEo3sfvO4eDYiILBORPe7l/Mmml95taEFQbSIio4AbgBnGmAlALfAfQDiw3RgzCfgMeNA9y9+Ae4wx44Bdjaa/DjxtjBmPdS3AMff0icCdWPemGAzMEJFewHeAMe7l/I93X6XSgqDa6gpgMvCliOxwx4OxLpX+h/s5rwGXikgU0NMY85l7+l+By0SkB9DfGPMvAGNMhTGmzP2cbcaYLGOMC2sYdyJQBFQAL4nId4G65yov0YKg2kqAvxpjJri/Rhhj/l8zz2ttLHxzl+TWqWz0cy0QZIypAaZiXRF6DbCqnTmrdtKCoNrqU+A6EekD9ffyS8Dahq5zP+dm4AtjTCFwWkS+4Z5+C/CZse73kCUi17iXESIiYS2t0H2viChjTBLW4cQEb7ww1SDI7gRU52CM2SMi9wNrRCQA64q7n2PduGSMiKQAhVjnGcC6FPc59x/8QeA29/RbgOdF5PfuZXyvldX2AN4XESfW3sWvPfyy1Bn0akd1XkSkxBgTYXceyjP0kEEpVU/3EJRS9XQPQSlVTwuCUqqeFgSlVD0tCEqpeloQlFL1tCAoper9f6A3+LK272cIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 norm of w: 17.363361\n"
     ]
    }
   ],
   "source": [
    "fit_and_plot(lambd=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yWGIF9oHSfKn"
   },
   "source": [
    "### 3.12.3.5 使用权重衰减"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 215
    },
    "colab_type": "code",
    "id": "EuQKKlUbtKJd",
    "outputId": "f7a82b55-9b47-4425-b987-0c169e9ae2a8"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAC1CAYAAACQyynAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAdsUlEQVR4nO2deZhU5ZX/P6fWXqGhAcVuZQkGFVCQ1kBwJholARVjEoNiMGZGZZzExNEs6phMYpJ54m+cJCYTI0MiMS7BcTQuKBGiRo3jCoRERAiIC83atHZD711V5/fHvUUVbXdTexVV5/M897n3vPe+7z21fO+73HcRVcUwjNLDk28HDMPIDyZ+wyhRTPyGUaKY+A2jRDHxG0aJYuI3jBLFl28HssGIESN07Nix+XbDMPLOmjVr9qrqyP7OFaX4x44dy+rVq/PthmHkHRF5Z6BzVuw3jBLFxG8YJYqJ3zBKlKKs8x+SSATC3eAvz7cnRpbp7e2lsbGRrq6ufLuSVcrKyqivr8fv9yccp+DFLyLnA+cAo4DbVHVVWgn2dMDPG+DkS+H06zLholHANDY2Ul1dzdixYxGRfLuTFVSV5uZmGhsbGTduXMLx8lLsF5GlIrJHRNb3CZ8jIptEZIuIXA+gqg+r6hXAF4EL0755oAJOugjqpqedlFH4dHV1UVtbW7TCBxARamtrky7d5KvOfycwJz5ARLzAbcBc4ARggYicEHfJt9zz6XPmv8GxZ2UkKaPwKWbhR0nlM+ZF/Kr6HPBen+BTgS2qulVVe4D7gE+Jw/8Dfq+qazPmRMd70LQpY8kZRn+0tLTwi1/8Iul4Z599Ni0tLVnwKEYhtfbXAdvi7EY37CvAWcAFInLlQJFFZJGIrBaR1U1NTYe+213nwWPXpOexYRyCgcQfDocHjbdixQpqamqy5RZQWA1+/ZVbVFV/BvzsUJFVdQmwBKChoeHQ0xPN/j6UDU3WR8NIiuuvv54333yTqVOn4vf7qaqqYvTo0axbt44NGzZw/vnns23bNrq6urj66qtZtGgREOul2tbWxty5cznttNN44YUXqKur45FHHqG8PP03VYUk/kbg6Di7HtiRTAIiMg+YN2HChENf/KEzkknaKAJuWv46G3bsy2iaJxw1hO/MmzTg+Ztvvpn169ezbt06nnnmGc455xzWr19/oFV+6dKlDB8+nM7OTk455RQ++9nPUltbe1AamzdvZtmyZfzyl79k/vz5PPjggyxcuDBt3wup2P8qcKyIjBORAHAR8GgyCajqclVdNHRogjn67g2w9u6kHTWMVDn11FMPeh33s5/9jJNOOokZM2awbds2Nm/e/IE448aNY+rUqQBMnz6dt99+OyO+5CXnF5FlwOnACBFpBL6jqneIyFXASsALLFXV15NMN/GcH+C1++Gl22HSpyFYlcytjMOQwXLoXFFZWXng+JlnnuHJJ5/kxRdfpKKigtNPP73f13XBYPDAsdfrpbOzMyO+5EX8qrpggPAVwIo00l0OLG9oaLgioQgzvgwzv2LCN7JGdXU1+/fv7/dca2srw4YNo6Kigo0bN/LSSy/l1LdCqvOnTdI5f1W/w5wNI2PU1tYya9YsJk+eTHl5OUccccSBc3PmzGHx4sWceOKJTJw4kRkzZuTUNynGefsbGho04fH8O/8Kz/8Ezv0JlGf31YqRe9544w2OP/74fLuRE/r7rCKyRlUb+ru+kBr80kZE5onIktbW1sQjhbrh7eeheUv2HDOMAqSoxJ90az9AfQN8baOzN4wSoqjEnxIi4PE6x0VYBTKMgSgq8adU7AfYvwtuPw1e+9/sOGYYBUhRiT+lYj9A5SioOQaC1dlxzDAKkKJ61ZcyHg8s+G2+vTCMnFJUOX/Kxf4ooR6nCmAYGSLVIb0At956Kx0dHRn2KEZRiT/lYn+UJafbMF8joxSy+K3YH89p10D5sHx7YRQR8UN6Z8+ezahRo7j//vvp7u7m05/+NDfddBPt7e3Mnz+fxsZGwuEw3/72t9m9ezc7duzgjDPOYMSIEfzxj3/MvHOqWnTb9OnT1TBUVTds2HBwwNKzVdfe4xyHehx73X2O3d3u2K894NidLY79+iOO3bbXsTeucOx9uw55/7feeksnTZqkqqorV67UK664QiORiIbDYT3nnHP02Wef1QceeEAvv/zyA3FaWlpUVXXMmDHa1NSU+mdVVWC1DqCToir2p13nB9i3EzamPLbIMAZk1apVrFq1imnTpnHyySezceNGNm/ezJQpU3jyySe57rrr+NOf/kTK1dZkGeipcDhvaeX8v79B9XsjVLv2pZ6GUTD0lxvmkvic/9prr9XFixf3e11zc7PefffdOmvWLL3ppptU1XL+3DPjSvjnF+ydv5ER4of0fvKTn2Tp0qW0tbUBsH37dvbs2cOOHTuoqKhg4cKFfP3rX2ft2rUfiJsNrMGvLzXH5NsDo4iIH9I7d+5cLr74YmbOnAlAVVUV99xzD1u2bOEb3/gGHo8Hv9/P7bffDsCiRYuYO3cuo0ePzkqDnw3p7Y/mN+HlxXDGjTbM9zDHhvSWyJDejNHZAmvvgp1/ybcnhpE1iqrYn/RMPgNRdzJ8Y4vV+42ipqhyfk23h18UERO+UfQUlfgTobWjlxWv7WT3vkMsatjVCnd/Bv58T24cM7JGMbZr9SWVz1hy4t/R2smX7l3L2nfeH/zC4BBAbYKPw5yysjKam5uL+gGg7hLdZWVlScUrqjp/ItRWBQDY294z+IUicMlDOfDIyCb19fU0NjaS0PqNhzFlZWXU19cnFafkxD+8whF/c1t3YhFUofN9qBieRa+MbOH3+w9aIceIUXLFfp/Xw7AKP81th8j5oyxbAPddnF2nDCMPFHzOLyLjgRuBoap6QSbSrK0K0tyeYM4/5QLo7XRKANLfQsKGcXiSl5xfRJaKyB4RWd8nfI6IbBKRLSJyPYCqblXVyzJ5/9rKAHsTzfmnXAAnX2LCN4qOfBX77wTmxAeIiBe4DZgLnAAsEJETsnHzEdXBxOv8AD3tsPkP2XDFMPJGXsSvqs8B7/UJPhXY4ub0PcB9wKeycf8RlQGaD9XaH8+aO+HeC2CvrepjFA+F1OBXB2yLsxuBOhGpFZHFwDQRuWGgyCKySERWi8jqQ73Wqa0K0tLRS284kphnU+bDFx+H4eMTu94wDgMKqcGvv0q1qmozcOWhIqvqEhHZCcwLBALTB7s2+q7/vfYejhiSQMeIqpG2oq9RdBRSzt8IHB1n1wM7kkkg0b79tZVBAPYmU+/vfB+euwV2v56MS4ZRsBSS+F8FjhWRcSISAC4CHk0mgUTn8BtRFe3ok0S9H+DZW5wVfQ2jCMjXq75lwIvARBFpFJHLVDUEXAWsBN4A7lfVpLLZhHP+KifnT/hdPzhTen9tI3zkn5JxyTAKlrzU+VV1wQDhK4CUp85NdDx/bao5v3XxNYqIQir2p02iOX910EfA60m8o088T/wrPP61FD00jMKhqMSfaJ1fRKitCiTX0ScWmf5fTBjG4UUhvepLG1VdDixvaGi44lDX1lYl2dEnyif/PQXPDKPwKKqcPxlqK5Ps4tuXtj2Zc8Yw8kBRiT+Z5bpqq5IY3NOXP98LP5oILe+mFt8wCoCiEn8yE3iOcIf1pjS907i/h9NvAH9lCl4aRmFQVHX+ZKitDNDVG6GjJ0xlMMmvoeZo+Ng3s+OYYeSIosr5kyv2Ox19mvanWO9XdXr7WXdf4zClqMSfTLF/ZHUK/fvjCXU503u9eFtq8Q0jz5RssX9kujm/vxwWPgSjSmMdOKP4SCjnF5GrRWSIONwhImtF5BPZdi6bRHP+pnRe99VPh0BFhjwyjNySaLH/H1V1H/AJYCTwD8DNWfMqRZKp8w+vDOCRNHL+KFuehAcug0iCE4MYRoGQqPij/VnPBn6tqn+hAPu4JlPn93qE2qpg+uJvb4Ydf4a2XemlYxg5JtE6/xoRWQWMA24QkWrgsM/qRmZC/FMugCmfA09RtZ0aJUCi4r8MmApsVdUOERmOU/Q/rBlZHUyvzg/g8Tr7SNiZ5bdsSPqOGUYOSDS7mglsUtUWEVkIfAs4dMW6wBlZnYGcHyAcgts+Ak/dlH5ahpEjEhX/7UCHiJwEfBN4B7gra17liJHVQfa2dROJpLmCq9cH0z4PH55z6GsNo0BIVPwhdTrBfwr4qar+FKjOnlu5YWRVkN6w0trZm35ip10Dx85OPx3DyBGJin+/O2f+JcDj7uo6/uy5lRrJvOqDDL3rj6enA15aDB191yMxjMIjUfFfCHTjvO/fhbPAxi1Z8ypFknnVB3Hiz0S9H+D9t+CJ6+BvT2QmPcPIIgmJ3xX8vcBQETkX6FLVoqjzQwbFf8QkuPL/YKot6W0UPol2750PvAJ8DpgPvCwiGVkuO59kXPwAR0529j0dmUvTMLJAosX+G4FTVPVSVf0CzqKa386eW7mhOugj6PNkrs4f5a3n4MfHw86/ZjZdw8ggiYrfo6rxk9Y1JxG3YBGRzL3rj+fIKTD+Y1CWWNuDYeSDRHv4PSEiK4Flrn0haSyukQwiUgn8AugBnlHVezOZflbEXz4M5h/2TSJGkZNog983gCXAicBJwBJVvS7Vm4rIUhHZIyLr+4TPEZFNIrJFRK53gz8DPKCqVwDnpXrPgchI//6B6GqFR78CjWuyk75hpEHCRXdVfVBVr1XVa1T1oTTveydwUHc4t+/AbcBc4ARggYicgLNa7zb3snCa9/0AI6uD7NnflelkHSJhp/6/Y2120jeMNBi02C8i+4H++r4KoKqa0igWVX1ORMb2CT4V2KKqW91734fTo7AR5wGwjiy0MxwxpIz3O3rpDoUJ+ryZTbxiOPzzizbhh1GQDComVa1W1SH9bNWpCn8Q6ojl8OCIvg74HfBZEbkdWD5QZBFZJCKrRWR1U1NTwjc9YkgWXvfFExX+7g1w57mwf3d27mMYSVJIc/j1NzmIqmo7CQwfVtUlIrITmBcIBKYnetNRQ8oA2L2vm/phWcyh25tg/04IdWbvHoaRBIX0uq4RODrOrgd2JJNAst17AY6odsS/Z1+W6v1Rxn8MvvQyDBvrTPv9/K2wL6mPZxgZpZDE/ypwrIiME5EAcBHwaDIJJDuwB2LF/t3ZFj84Q38BmjbB0z+ATb937N5O6xFo5Jy8iF9ElgEvAhNFpFFELlPVEHAVsBJ4A7hfVZNaESOVnH9YRQC/V9idrTp/f4w6Dr6yBqZd4tivPww3HwPNbzr27g2w4RFnkhBwHgzdbU6JwRgcVed7i4RjdncbhNzfNxJx5l2MPmwjYWhthO79jh0Owd4t0Nni2KEe2PVabKRmbydse8VJA5y0tz4LbW47U2cL/G1lbCHX9mZ4/aFYW8/+3bBuWcxubYRX74hd/95WZy2IqN30N3jullj6u16Dp74fu38a5EX8qrpAVUerql9V61X1Djd8hap+WFU/pKpJr4WdSs7v8Qijqstyk/PHM2wM+ALO8ZGTYdZXYdg4x97wMNz/hdi1f/qR83AQt1nkqe/Dj46LnX/6B3DbjJj95E3wy4/H7D/8G/z6nJi98ka4+zMx+/fXwbIFMfvxrx98/0e/Cg9eHrMf/pKzRXnwcqc/Q5T/WQiPXROz753vpBnlN/Oce0b51Wx44oaYffssx8coP50Kq+J6k98yAZ78bsz+99Hw1Pdi9vdr4dn/cI7DvfDDOnjhvxy7tx1uGQ+r73Dsrlb4ySRY91vHbm+Cn093BAtOO83i02KltNZGuGM2vPm0Y7e8A3edB+++4NjvbYXfzoftbt+O5i3wv1+E3W6Xlr2b4OErYe/fHLtpIzx+Lbz/tmtvgpX/Cvu2u/Ybzu/bvid2/vkfQ2f6w8YLqcEvbVR1ObC8oaHhimTijRoSZM++HOb8fTlyirNFmXkVHH9erJow4SzntWGUo6ZBOG6F4RETYfz+mF37IWc+wSg1Y2I5H0DNMbGcEWBoPUjca84ho51FSeLP93YebMeXQmrGgC8Ys4d/CMprYvao46DqyJg9eqrjQ5QxM504USacCaMmxexJ58Pok2L2yV+A+lNi9syr4Bj34ScCZ9wIYz7q2B4vzP4eHOPavjKYewsc8xHHDlTCef8VS6+8Bj7zS6hz24wrR8L8u+Goqe53cxR8/kFnBCc4bThffBxGug/jkRPhiqdjn+fIyc7r3ujnrT8FvvpnqB7tfvbT4GuboNz9fSecBde/C4Eqxz7uXPhWE3jd6TOmXOBsGUBSWqW2QBGRecC8CRMmXLF58+aE4/3T3avZ2tTOH679WPacM4w8ICJrVLWhv3OF1OCXNqnU+cHp6LMnl3V+wygAikr8qXLEkDJaO3vp6s1472HDKFiKSvypNPgBjHIn9chrvd8wckxRiT+dYj/A7mwN8DGMAqSoxJ8qB8Sf69d9hpFHTPzE9/KzYr9ROhSV+FOt8w8t9xPwebLfv98wCoiiEn+qdX4RYVR10Ir9RklRVOJPh9FDy9jRauI3SgcTv0tdTTnb37ex9kbpUFTiT7XOD1A/rIJd+7oIhSNZ8MwwCo+iEn+qdX6AumHlhCPKLqv3GyVCUYk/HeqHOaPYGq3ob5QIJn6XuhpH/FbvN0oFE7/LUTWW8xulhYnfpczvZWR1kO0tNpeeURoUlfjTae0Hp95vOb9RKhSV+NNp7Qf3XX+Lid8oDYpK/OlSP6yCHS2dRCLFM7WZYQyEiT+OumHl9IbVpvQySgITfxzRd/3W6GeUAib+OOrtdZ9RQhS8+EVkvIjcISIPZPtedW7O/26z5fxG8ZNV8YvIUhHZIyLr+4TPEZFNIrJFRK4fLA1V3aqql2XTzygVAR9HDy/njV37cnE7w8gr2V6x507g58Bd0QAR8QK3AbNxVuZ9VUQeBbzAD/vE/0dV3ZNlHw9iSt1Q1m838RvFT1bFr6rPicjYPsGnAltUdSuAiNwHfEpVfwicm01/EmFy3VBWvLaL1o5ehlb48+2OYWSNfNT564BtcXajG9YvIlIrIouBaSJywyDXLRKR1SKyuqmpKWXnJh/ldBBavyO1XoKGcbiQD/FLP2ED9qpR1WZVvdJdubdvtSD+uiWq2qCqDSNHjkzZuSl1rvi3m/iN4iYf4m8Ejo6z64EdmUg43b79AMMqA9TVlPOaid8ocvIh/leBY0VknIgEgIuAR/Pgx4A4jX4mfqO4yfarvmXAi8BEEWkUkctUNQRcBawE3gDuV9XXM3G/dAf2RJlcN4S3mzvY19WbCbcMoyDJdmv/ggHCVwArMn0/EZkHzJswYUJa6Ux26/1r33mf0yeOyoBnhlF4FHwPv2TIVM4/Y3wttZUB7nrxnQx5ZhiFR1GJP1OU+b1c+tGxPL1xD5t27c+3O4aRFYpK/Jlo7Y9yyYwxlPu9LHluawY8M4zCo6jEn6liPziv/C485WgeXredO//vLVRtgg+juMh23/6ckqkGvyjXzP4w777XwXeXb2D5X3dyxsSRHHfkEI4cWkZtVYCa8gBlfg8i/fVbMozCRooxR2toaNDVq1dnJC1V5a4X3+G3L7/Lpt0frP8HvB6GVvgZWu5sNe5+SHksLN4eUu5jSJljVwa89uAwsoqIrFHVhn7PmfgTp6Wjh7ebO9jV2sX7HT28197Dvs5eWvvbOnrZ3x0aND2PQFXQR3WZn+qy6EPBsSuDXqrL/O55H5UBH5XR46CPqqCXyqBzXOH34vMWVQ3OyBCDib+oiv3ZpqYiwNSKwMGdkwchHFH2d8UeCPu7QnHHvezrDDn7rth+R0sXbd1t7O9yrg8lOJlo0OehKuijIuilMuCjIuClIuCjPOB1j72U+53w8oCXcr/3wLkyf8wu83kpD3gI+pzwMr+HMr8Xvz1cio6iEn+m6/zp4vUINRUBaioCKcVXVbpDEdq7Q+zvCtHWHaK9O0R7T4i27rBz3O2Ed/aED973hunoCbO3rZuOHue4qzdMR0+IVCYn9nqEMp/HfSB4Cfo8BFw76PMQjO7d8KAvdk3AGwsPxIVF936vB3+fMJ9X8Hvd8z7n2O9xjn0eD36vWJUpTazYX2KoKj3hCJ3uA6GzN0yn+2Do7A3T1Rtx9/FbhK7eMN2hAfa9EbrDEbp7w/SEI44ditATcu1QhGz8zbwewe8V/B7nYeHzevB7nL3PK/g8sQeFz+vB63HConuf13OQ7fW4tlfwihx0ffzm8wgeT+war0fwxB17xT3vIRYuzsPK2yfcI4IIB+J4RPDIwec97vXR8Ojm9wmjh5YP+h1Zsd84gIi4ubKXmorc3FNVCUWUnlDE2cIH73vDztYdihAKO9eFIo7dG9YD56PHIfc4FIkPi9mhcIRQRN0wJzzkXtcTihDWuHPhmB12rw1HosdKJLp3P0Mh5ZXHDK/guW+ekXJ8E7+RdUTcHNrroTKYb2/SI/5hEH1AqHscju4jSiTCATuisevjw/XAeQ66Rt37xJ9TdeLpgbhQEfCm9VmKSvyFVuc3ig+PRwh4iqOtoaiacDPZw88wip2iEr9hGIlj4jeMEsXEbxglSlG+5xeRJmCgmThGAHtz6M6hMH8GppB8gcPTnzGq2u901kUp/sEQkdUDdXrIB+bPwBSSL1B8/lix3zBKFBO/YZQopSj+Jfl2oA/mz8AUki9QZP6UXJ3fMAyHUsz5DcOghMQvInNEZJOIbBGR6/Nw/6NF5I8i8oaIvC4iV7vhw0XkDyKy2d0Py7FfXhH5s4g8lm9/RKRGRB4QkY3u9zQzX/6IyDXu77ReRJaJSFmufRGRpSKyR0TWx4UN6IOI3OD+vzeJyCcPlX5JiF9EvMBtwFzgBGCBiJyQYzdCwNdU9XhgBvBl14frgadU9VjgKdfOJVfjLJsWJZ/+/BR4QlWPA05y/cq5PyJSB3wVaFDVyYAXZ03JXPtyJzCnT1i/Prj/pYuASW6cX7j/+4FRd5hgMW/ATGBlnH0DcEOefXoEmA1sAka7YaOBTTn0od79A30ceMwNy4s/wBDgLdx2qLjwnPsD1AHbgOE4I18fAz6RJ1/GAusP9X30/U/jrIU5c7C0SyLnJ/ZjRml0w/KCiIwFpgEvA0eo6k4Ad5/LxQFvBb4JROLC8uXPeKAJ+LVbDfmViFTmwx9V3Q78J/AusBNoVdVV+fClHwbyIen/eKmIv78B2Hl5zSEiVcCDwL+o6r58+OD6cS6wR1XX5MuHPviAk4HbVXUa0E7uq0AAuPXoTwHjgKOAShFZmA9fkiDp/3ipiL+Rg+fcrQd25NoJEfHjCP9eVf2dG7xbREa750cDe3LkzizgPBF5G7gP+LiI3JNHfxqBRlV92bUfwHkY5MOfs4C3VLVJVXuB3wEfzZMvfRnIh6T/46Ui/leBY0VknIgEcBpGHs2lA+JMNXsH8Iaq/jju1KPApe7xpThtAVlHVW9Q1XpVHYvzfTytqgvz6M8uYJuITHSDzgQ25Mmfd4EZIlLh/m5n4jQ+5uW76cNAPjwKXCQiQREZBxwLvDJoSrlozCmEDTgb+BvwJnBjHu5/Gk4x7K/AOnc7G6jFaXTb7O6H58G304k1+OXNH2AqsNr9jh4GhuXLH+AmYCOwHrgbCObaF2AZTptDL07OftlgPgA3uv/vTcDcQ6VvPfwMo0QplWK/YRh9MPEbRoli4jeMEsXEbxglionfMEoUE7+RU0Tk9OgIQiO/mPgNo0Qx8Rv9IiILReQVEVknIv/tjvtvE5EfichaEXlKREa6104VkZdE5K8i8lB0jLmITBCRJ0XkL26cD7nJV8WN27/X7UWHiNwsIhvcdP4zTx+9ZDDxGx9ARI4HLgRmqepUIAx8HqgE1qrqycCzwHfcKHcB16nqicBrceH3Arep6kk4feN3uuHTgH/BmVthPDBLRIYDnwYmuen8ILuf0jDxG/1xJjAdeFVE1rn2eJyhv//jXnMPcJqIDAVqVPVZN/w3wN+LSDVQp6oPAahql6p2uNe8oqqNqhrB6eY8FtgHdAG/EpHPANFrjSxh4jf6Q4DfqOpUd5uoqt/t57rB+oYPto51d9xxGPCpagg4FWfU4/nAE0n6bCSJid/oj6eAC0RkFByYN24Mzv/lAveai4HnVbUVeF9E/s4NvwR4Vp25ChpF5Hw3jaCIVAx0Q3eeg6GqugKnSjA1Gx/MiOHLtwNG4aGqG0TkW8AqEfHgjCr7Ms4EG5NEZA3QitMuAM7Q0sWuuLcC/+CGXwL8t4h8z03jc4Pcthp4RETKcEoN12T4Yxl9sFF9RsKISJuqVuXbDyMzWLHfMEoUy/kNo0SxnN8wShQTv2GUKCZ+wyhRTPyGUaKY+A2jRDHxG0aJ8v8BYeB5qlWUvy4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 norm of w: 0.31498882\n"
     ]
    }
   ],
   "source": [
    "fit_and_plot(lambd=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xJiYKczaS3PS"
   },
   "source": [
    "## 3.12.4 简洁实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mJrspxkzSvAM"
   },
   "outputs": [],
   "source": [
    "def fit_and_plot_tf2(wd, lr=1e-3):\n",
    "    net = models.Sequential()\n",
    "    net.add(layers.Dense(1))\n",
    "    net.build(input_shape=(1, 200))\n",
    "    w, b = net.trainable_variables\n",
    "    optimizer = optimizers.SGD(learning_rate=lr)\n",
    "    train_ls, test_ls = [], []\n",
    "    for _ in range(num_epochs):\n",
    "        for X, y in train_iter:\n",
    "            with tf.GradientTape() as tape:\n",
    "                l = loss(net(X), y) + wd * l2_penalty(w)\n",
    "            grads = tape.gradient(l, net.trainable_variables)\n",
    "            # 对两个optimizer实例分别调用assign_gradients函数，从而分别更新权重和偏差\n",
    "            optimizer.apply_gradients(zip(grads, net.trainable_variables))\n",
    "        train_ls.append(tf.reduce_mean(loss(net(train_features),\n",
    "                             train_labels)).numpy())\n",
    "        test_ls.append(tf.reduce_mean(loss(net(test_features),\n",
    "                            test_labels)).numpy())\n",
    "    semilogy(range(1, num_epochs + 1), train_ls, 'epochs', 'loss',\n",
    "                 range(1, num_epochs + 1), test_ls, ['train', 'test'])\n",
    "    print('L2 norm of w:', tf.norm(w).numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAC1CAYAAAC56YFuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAfDElEQVR4nO3dd3yVhdXA8d9JCImBJJCQsAKEvZcECuIABwJurbP0ba11tPW12moVraXahbVDbRVFxa3VOl4VERCqgApiYplhBEKAQAgQyN7Jef94bm4CZpOb5yY538/nfsh57jPOXYdnH1FVjDEGIMDtBIwx/sMKgjHGywqCMcbLCoIxxssKgjHGywqCMcarg9sJtIRu3bppXFyc22kY4xcSExOPqmp0Tc+1i4IQFxdHQkKC22kY4xdEZG9tz9kmgzHGywqCMcbLCoIxxssKQqX//B5W/6Va/Af48p9V8YrfwroFVfGyB2D9s1Xxx/dBwqKq+KO74ZuXq+IP74QNr1fF798Om96qit+7Dba8WxW/czMkfeD8XV7mxNs/cuLSQifeudyJi3OdeNdKJy487sQpq5w4/6gTp37hxLmHnHjfV06cnebEaYlOfDzViQ9ucOKju5w4Y6sTH9nhxEd2OHHGVic+usuJD25w4uOpTpyW6MTZaU687ysnzj3kxKlfOHH+USdOWeXEhcedeNdKJy7OdeKdy524tNCJt3/kxOVlTpz0gRNX2vKu8/5W2vSW8/5X2vC68/lU+uZl5/OrlLDI+XwrrX/W+fwrrVsAn8yrir/8B6z8XVW85m/w2fyqePWjzqPSZ/OdcSqt/J0zj0qfzGvcd+8UWEGolLkbju+pig9thiPbT4yPJlfFGVucabzPb4Jj1aZP31j1gwA4+F/I2ld7fCCx6gcDkPY15Bx0/tYKJ85Nd+KKcifO8/ygKso88WEnLi914vwjTlxW7MQFnh9cWZEnznTi0kInrvwBlhQ4cVGWJ87zxNlOXJzrxJU/0KJsJy7J88RZnrjAiQuPO3HlD7gg04nLijzxUU9c7MT5R5y4vNSJ8w47cYXnB593yBOXO3FuuhNrhRPnHHTiStlpzvtbKWuf8/7XFh9PdT6/Ssf2OJ+vN05xvg+VMndVFUtwvieHk2qPj+yoKqbgPFf9u3VynLHVWUalQ5udHLzxSd+9UyDt4WrH+Ph4taMMxjhEJFFV42t6ztYQjDFeVhCMMV5WEICcolLWJB9xOw1jXGcFAfjrsh38+KUEsgpK3E7FGFdZQQCum9SX4rIK3k5Mq39kY9qwVlkQRGSAiDwvIm83x/yG9wxnQr+uvP7VPtrDURdjatPiBUFEFonIYRHZctLwmSKyQ0R2ich9tU0PoKopqnpTc+b1ve/0JeVoPmtTMptztsa0Km6sIbwIzKw+QEQCgSeBWcAI4HoRGSEio0Vk8UmPGF8kNXt0T7qEBvHqulovBDOmzWvxy59VdbWIxJ00eBKwS1VTAETkX8Blqvon4OKmLEdEbgFuAejbt2+944cEBXLdxL4sXL2bfZkF9I0KbcpijWnV/GUfQm9gf7U4zTOsRiISJSJPA+NFZG5N46jqQlWNV9X46Oga7wXxLTdOjSMwQHj+85T6RzamDfKXgiA1DKt1756qZqrqbao60LMW0Sy6h4dw+bjevJmwn2P5dgjStD/+UhDSgD7V4ljgoBuJ3HL2AIpKK3jpy1Q3Fm+Mq/ylIHwNDBaR/iLSEbgO+MCNRAZ3D+P84d158ctUcotK3UjBGNe4cdjxDWAtMFRE0kTkJlUtA24HlgHbgLdUdWtd8/Gl/z13ENmFpbxiRxxMO+PGUYbraxm+BFjSwunUaGyfLpw9JJrn1uzhh2fEEdqxXdyL1hi/2WTwO3ecO4hj+SV2XoJpV6wg1CI+LpKzBnfjmVUpFJSUuZ2OMS3CCkId7jx/CJn5Jby81tYSTPtgBaEOE/p1ZdrQaJ5ZtZu8YltLMG2fFYR6/OKCIRwvKGXR581zE0tj/JkVhHqMie3ChSO78+zqFI7b2YumjbOC0AC/nDGUvJIynl69u/6RjWnFrCA0wJDuYVwxrjcvfpHKoewit9MxxmesIDTQXRcMoUKVx1fudDsVY3zGCkID9YkMZc7kfrz59X52Hc5zOx1jfMIKQiPcPn0QoR078Oiy7fWPbEwrZAWhEaI6B3PL2QNYtjWDxL3H3E7HmGZnBaGRfnxWf6LDgvnjku12h2bT5lhBaKTQjh34xQVDSNx7nGVbD7mdjjHNqtUWBBHpJCKJItKkm7CeiqsnxDIopjOPLN1BaXlFSy/eGJ9plX0ZPO4F3vJNlnXrEBjA/bOHsedoPq9/tc+NFIzxiVbZl0FEzgeSgIyWTr7S9KExnDEwisdW7CS70G61ZtqGFi8IqroaOHkXvbcvg6qWAJV9GTar6sUnPQ4D04HJwA3AzSLixpoO988eTlZhKU99tqulF2+MT/jLPoRG9WVQ1QdU9U7gdeBZVf3WhryI3CIiCSKScOSIb1q9j+odwZXjY3nh81T2HyvwyTKMaUn+UhAa1ZfBO4Lqi6q6uJbnGt2opSnuuXAoAQHwyFI7Wcm0fv5SEPymL0Nj9YgI4ZazBrB4UzqJe4+7nY4xp8RfCoLf9GVoilvPGUhMWDC/W5xERYWdrGRaL+vL0Aw6BXfg7guHsmF/Fh9uahUrNsbUSNrD6bfx8fGakJDg02VUVCiX/PNzjueXsPKX0zitY6BPl2dMU4lIoqrG1/Scv2wytHoBAcKDF4/gYHYRz62x7tGmdbKC0IwmD4hi5sgeLFi1m4wcu7OSaX2sIDSzubOHUVauPLpsh9upGNNoVhCaWb+oTtw4NY63E9PYnJbtdjrGNIp1MfWBn507iLcT03h48VbeunUKIjWdd2XcUFpaSlpaGkVFbX+TLiQkhNjYWIKCgho8jRUEHwgPCeKXM4Zy/3ub+WhzOheP6eV2SsYjLS2NsLAw4uLi2nShVlUyMzNJS0ujf//+DZ7ONhl85NqJfRjWI4w/LdlOUWm52+kYj6KiIqKiotp0MQDn4ruoqKhGrwlZQfCRwADhNxeP4EBWIc9bGzi/0taLQaWmvE4rCD50xqBuzBjRnSc/3cVhOwxpWgErCD72wEXDKStX/myHIQ2QlZXFU0891ejpZs+eTVZWlg8yOpEVBB+rfhhyU5rvP1Dj32orCOXlde9nWrJkCV26dPFVWl5WEFrA7ecOolvnjjz8YZLdur2du++++9i9ezfjxo1j4sSJTJ8+nRtuuIHRo0cDcPnllzNhwgRGjhzJwoULvdPFxcVx9OhRUlNTGT58ODfffDMjR45kxowZFBYWNlt+dtixBYSFBHH3jKHc9+5mFm9K55KxdhjSHzz04VaSDuY06zxH9Apn3iUja31+/vz5bNmyhQ0bNvDZZ59x0UUXsWXLFu+hwUWLFhEZGUlhYSETJ07kqquuIioq6oR5JCcn88Ybb/Dss89yzTXX8M477zBnzpxmyd/WEFrI1fF9GNEznPkf22FIU2XSpEknnCfwxBNPMHbsWCZPnsz+/ftJTk7+1jT9+/dn3LhxAEyYMIHU1NRmy6dBawgi8nPgBSAXeA4YD9ynqsubLZM2LjBA+M0lI7hu4ToWrk7hjvMGu51Su1fX/+QtpVOnTt6/P/vsM1asWMHatWsJDQ1l2rRpNZ5HEBwc7P07MDCwWTcZGrqG8CNVzQFmANHAjcD8ZsuikUTkLBF5WkSeE5Ev3cqjsSYPiGLWqB4s+Gw36dnN9yGa1iMsLIzc3Nwan8vOzqZr166Ehoayfft21q1b18LZNbwgVJ7hMBt4QVU3UvONUeufUTM0alHVNap6G7AYeKkpebjl/tnDKVflz0vtMGR7FBUVxdSpUxk1ahT33HPPCc/NnDmTsrIyxowZw4MPPsjkyZNbPkFVrfeBs7mwHEgGQoEwILEh09Ywr7OB04Et1YYFAruBAUBHYCNOw5bROD/66o+YatO9BYTXt8wJEyaoP3nk423a797Fmrj3mNuptDtJSUlup9Cianq9QILW8ltp6BrCTcB9wERVLQCCcDYbmlKAmqNRCyLSF8hWZ1OmVfnp9EFEhwXbYUjjdxpaEKYAO1Q1S0TmAL8GmvNi/0Y1avG4CWfNpUYt0ailqToHd+BXnpuyvr/Bbspq/EdDC8ICoEBExgK/AvYCLzdjHo1u1KKq81S11h2K2kKNWprqqtNjGRMbwfyPt1NQUuZ2OsYADS8IZZ5tj8uAx1X1cZz9CM2l1TZqaaoAz9WQh3KKeHqV3ZTV+IeGFoRcEZkLfB/4yNOtueG3Yalfq27U0lTxcZFcMrYXz6zazYEsOwxp3NfQgnAtUIxzPsIhnO37R5uywLbYqOVU3DdrGADzP7bekMZ9DSoIniLwGhAhIhcDRarapH0Iqnq9qvZU1SBVjVXV5z3Dl6jqEFUdqKp/aMq8W6PeXU7j1rMH8OHGgySknnzwxbQ1Tb38GeCxxx6joMC3XcYbVBBE5BpgPXA1cA3wlYh815eJtSe3TRtIj/AQHvrQekO2df5eEBp6teMDOOcgVJ4DEA2sAN72VWLtSWjHDtw3axh3vrmBd75J4+r4PvVPZFql6pc/X3DBBcTExPDWW29RXFzMFVdcwUMPPUR+fj7XXHMNaWlplJeX8+CDD5KRkcHBgweZPn063bp149NPP/VJfg3dhxBQWQw8MhsxrWmAy8b1YnzfLvx52Q7yiu0wZIt54SL472vO3+WlTrzxTScuKXDiLe84cVG2Eyd59nfnZzrxjo+dODej3sXNnz+fgQMHsmHDBi644AKSk5NZv349GzZsIDExkdWrV7N06VJ69erFxo0b2bJlCzNnzuSOO+6gV69efPrppz4rBtDwH/VSEVkmIj8UkR8CHwFLfJZVOyQizLtkJEdyi3nq011up2NawPLly1m+fDnjx4/n9NNPZ/v27SQnJzN69GhWrFjBvffey5o1a4iIiGixnBq0yaCq94jIVcBUnJOIFqrqez7NrB0a16cLV47vzXOf7+H6SX3pExnqdkpt340fVf0dGHRi3DH0xDgk4sS4U9SJcVj3Ri1aVZk7dy633nrrt55LTExkyZIlzJ07lxkzZvCb3/ymUfNuqgav9qvqO6r6C1W9y4qB7/xq5jACRfjjkm1up2J8oPrlzxdeeCGLFi0iLy8PgAMHDnD48GEOHjxIaGgoc+bM4e677+abb7751rS+UucagojkUvMpxAKoqob7JKt2rEdECD+ZNpC/fbKTdSmZTB4QVf9EptWofvnzrFmzuOGGG5gyZQoAnTt35tVXX2XXrl3cc889BAQEEBQUxIIFCwC45ZZbmDVrFj179vTZfgRpD1fbxcfHa0JCgttpNFhRaTnn/XUV4acFsfh/zyQwoH00FmkJ27ZtY/jw4W6n0WJqer0ikqiq8TWNb0cK/FBIUCBzZw9jW3oOb369v/4JjGkmVhD81EWjezIpLpK/Lt9BTlGp2+mYdsIKgp8ScW7KeqyghH+s/Padd03TtYfNZGja67SC4MdG9Y7g6gmxvPhlKilH8txOp00ICQkhMzOzzRcF9bSDDwkJadR01qjFz9194VCWbD7EH5ds47kfTHQ7nVYvNjaWtLQ0/O0uWr4QEhJCbGxso6axguDnYsJC+Nn0QTyydDtrko9w1mD/u/tTaxIUFHRCYxRzIttkaAV+dGYcfSND+d3iJMrKK9xOx7Rhfl8QRGSAiDwvIm9XG9ZJRF4SkWdF5Htu5tcSgjsEcv/s4ezMyOP19fvcTse0YT4tCM3UlCVFVW86afCVwNuqejNwaTOn7ZcuHNmdKQOi+NsnO8kusMOQxjd8vYbwIjCz+gDP/RifBGbhNGO5XkRGiMhoEVl80iOmlvnGUnXb9nbRObXyMGROYSmPrdzpdjqmjfJpQWiupiw1SMMpCtAKNnuay/Ce4Vw3qS+vrN3LbjsMaXzAjR9To5qyiEiUiDwNjPfc+RngXeAqEVkAfFjLdH7bqOVU/OKCIYQEBfKnJXZTVtP83Djs2KimLKqaCdx20rB86mklp6oLgYXgXNzU+DT9U7fOwfxk2kAeXbaDtbszmTLQroY0zceNNYR215Slud10Zn96dzmN33+URLndlNU0IzcKQrtsytKcQoICuXfWMLYezOGtBLsa0jQfXx92tKYsPnLJGOdqyEeX7SC70A5Dmubh66MM1pTFR0SEeZeOIKughMdX2NWQpnm0m0N2bdHIXhFcO7EPr6xLJfVovtvpmDbACkIrd9f5QwgKDOCRpXYY0pw6KwitXEx4CLeePZCPtxxi/R7rDWlOjRWENuDms/vTKyKEX//fZkrK7GpI03RWENqA0I4dePiyUezMyOPZNSlup2NaMSsIbcT5I7oza1QPnliZbDsYTZNZQWhDfnvpSDoGBnD/e5vb/D0DjW9YQWhDuoeHMHf2cL7cncm/E9PcTse0QlYQ2pjrJvZhUlwkf/hoG1kFJW6nY1oZKwhtTECA8PDlI8kpKuVJaytvGskKQhs0rEc43z09lpe+3Mv+YwVup2NaESsIbdQvZgxBBObbGYymEawgtFE9I07jp9MG8dGmdJZsTnc7HdNKWEFow346fSCje0fwwHubOZxT5HY6phWwgtCGBQUG8Pdrx1JQUs6D72+pfwLT7vl9QailUctwEXlaRN4WkZ+4mZ+/GxQTxh3nDWbZ1gxWJGW4nY7xc62yUYuqblPV24BrgPjmz7xtufmsAQyO6cy8D7ZakxdTp9baqAURuRT4HFjpu/Tbho4dAvjjlaNJzy7kjPkr+e0HW8kvLnM7LeOHWmujFlT1A1U9A2jzvR2bw8S4SD64/Uxmje7JK+v2cu3CtRzOtR2N5kStslGLiEwTkSdE5BlgSS3TtclGLadiVO8I/nL1WJ77n3h2H87nuwvWciS32O20jB9xoyA0ulGLqt7muSHrnzzDPlPVO1T1VlV9spbpFqpqvKrGR0dHN1PqbcP0YTG8dvN3OJxbxI9fTqCwpJzisnbRItPUwxq1tFOn9+3KE9eNZ1NaFmMeWsbQXy/l7n9vtMYv7Zwbrdy8jVqAAziNWm5wIY92b8bIHjwzZwJf7TlGXlEZbybsp0OA8McrRhMQUNOKnGnrfFoQPI1apgHdRCQNmKeqz4tIZaOWQGCRNWpxz4yRPZgxsgcAMeHB/OM/u1iXksn/TIljUv9IhnQPo2MHvz9dxTQTaQ931omPj9eEhAS30/B7qsrHWw7xzOoUNu7PApzmsvMuGcHFY3oiYmsNbYGIJKpqjefvWEEw36Kq7M0sYNOBbJ5bk8KmtGwmD4jk1rMHcubgbgQFBrD/WAGbD2STV1RGdHgw5wyO9slmRnp2IYs3pnMgq5BZo3owqX+kFaZTZAXBCkKTlVcor6xN5elVKRzKKSIwQOga2pGjeScerhzWI4xLxvYiqlNHMvNLyMwr4ZKxPRnftyuZecVsOZjDvsx8hnQP4zsDnBb2FRXKjoxcNh/I5mBWIcfzSygpr2BwTBhnD4lm0Rd7eGP9PlSdk6tKyiroFRHCOUOjOW9Yd84c3I2QoMBGvZ684jKWbE6nsKScCf26MrRHGEGB/rVJVFRazjOrUtiRkcP9s4cT2zUUgJQjeTz0YRJXnt6by8bVeqS+XlYQrCCcstLyClYkZbD1YA4HswsZ1SuCSf0j6RIaRELqcf756S52Hc7zjt8xMICS8gr6Roay76SbtEweEElwh0A2pWVx3HMqtQiEhwQRFCgczXNu/RYYIHx/cj9+cEYc3cOD+XjzIT5JyuCLXUfJLS6jc3AHfnBGP247ZyBhIUH1voaXvkzlkaXbKSipOsTaMTCAIT06M3NkD644PZbeXU7zPnc4p4j3/nuA80d0Z2B051N6/xpqx6Fcbnrpa9KOF9KxQwDBHQK4ffogBsV05t53NpOZX4wqnDW4G6pQUFLG4JgwRsVGMGVAJAOjO9e7BmUFwQpCiygsKSczv5iuoR1R4JW1e1m/J5MJ/boSHxdJn8hQPt6czgtfpBIW0oHRvZ2iMqFfV2K7hnp3Xu7MyGX1ziOcNTiaoT3CvrWckrIK1qZk8u+E/SzelE54SAfi4yIZExvBmNgIJsZFfqtALPp8Dw8vTuKcIdHccd5guocHk7j3OEkHc0jce5yEvcfpGBjACzdO5Dv9I/nTx9t5eW0qpeVKz4gQ3vvpVHpEhJwwzwrPIdrqm0rlFcq+YwVsT88hIEA4Y2BUg4oVQFZBCZf+8wsKS8t5/Lpx9Okayi/f2sj6VOdk3+7hwbz8o++wfOshXl+/j5jwEEI6BJB8OI9j+U4R/fGZ/fn1xSPqXI4VBCsIbdamtCxeXruXjfuz2HUkD1Xnh/PM9+MpLClnward7MrI5WB2ETNH9uAfN4yvcRNhX2YBN7+cwIGsQsb37cKa5KNcG9+H80d05643NxDb9TRuP3cQg2PC6BIaxIptGfx1+U46BgZwTXwsWYWlrN9zjJQj+ZSUV3XPCgoU7p4xlFvPGQg4myyb9meRlJ7DvmMF7Dmaz+7DeUR1DkZRdhzK5V+3TGZCv0jvPA5kFbI5LYvxfbvSPTzkW7mrOkVoXUomg2LCmNCva53vmRUEKwjtQl5xGd/sPc79720mPbuI8grnf/fJA6IY1iOMG6f2r/MQanp2IVc+9SWHcoqYd/EIfji1PwCrdx7htlcTT9jUAJgUF0lwUABrko9yWlAg8XFdGdEznIHRnRneM5z8kjKeWbWb1clHef9nU0k+nMuv3t5EabnzmwsL7kBct04MjO7EwawiNh3IYt4lI7l+Ul/fvUlYQbCC0M4czy/h4cVJDIzuxI/PGtCoHY8Hsgo5lF14wv/QAMVl5SRn5JFyNJ+cwlJ6dQlh+tAYRISjecWEhwTVWGyyC0o5/++r6BgYQHp2IZP6R/KTaYMY3TuCrqFBrhwxsYJgBcG4aOmWQ9z2aiIT+nXl5R9NolOwGycIV6mrILibmTHtwMxRPfj3bVMY0TPc9WJQH//Ozpg2YmJcZP0j+QH/OiPDGOMqKwjGGC8rCMYYLysIxhivdnHYUUSOAHtrebobcLQF06mP5VM3y6duDcmnn6rWeF/BdlEQ6iIiCbUdk3WD5VM3y6dup5qPbTIYY7ysIBhjvKwgwEK3EziJ5VM3y6dup5RPu9+HYIypYmsIxhivdl0QGtOF2kfL7yMin4rINhHZKiI/9wyPFJFPRCTZ82/dd7xo3pwCReS/IrLYD3LpIiJvi8h2z3s0xeV87vJ8TltE5A0RCWnJfGrqpl7X8kVkrue7vUNELmzIMtptQaitC3ULp1EG/FJVhwOTgZ95crgPWKmqg3G6W7dksfo5sK1a7GYujwNLVXUYMNaTlyv5iEhv4A4gXlVH4fQUua6F83mRk7qp17Z8z/foOmCkZ5qnPN/5uqlqu3wAU4Bl1eK5wFyXc3ofuADYAfT0DOsJ7Gih5cd6vlTnAos9w9zKJRzYg2c/V7XhbuVT2aQ4Eucq4cXAjJbOB4gDttT3fpz8fcZpjDSlvvm32zUEGtmF2tdEJA4YD3wFdFfVdADPvzEtlMZjwK+AimrD3MplAHAEeMGzCfOciHRyKx9VPQD8BdgHpAPZqrrcrXyqqW35Tfp+t+eC0Kgu1L4kIp2Bd4A7VTXHpRwuBg6raqIby69BB+B0YIGqjgfyadnNlRN4ts0vA/oDvYBOIjLHrXwaoEnf7/ZcEPyiC7WIBOEUg9dU9V3P4AwR6el5vidwuAVSmQpcKiKpwL+Ac0XkVZdyAefzSVPVrzzx2zgFwq18zgf2qOoRVS0F3gXOcDGfSrUtv0nf7/ZcELxdqEWkI84OmA9aMgFx7rD5PLBNVf9W7akPgB94/v4Bzr4Fn1LVuaoaq6pxOO/Ff1R1jhu5ePI5BOwXkaGeQecBSW7lg7OpMFlEQj2f23k4OzndyqdSbcv/ALhORII9ndYHA+vrnVtL7JDx1wcwG9gJ7AYecGH5Z+Ksxm0CNnges4EonJ17yZ5/I1s4r2lU7VR0LRdgHJDgeX/+D+jqcj4PAduBLcArQHBL5gO8gbP/ohRnDeCmupYPPOD5bu8AZjVkGXamojHGqz1vMhhjTmIFwRjjZQXBGONlBcEY42UFwRjjZQXBuE5EplVeXWncZQXBGONlBcE0mIjMEZH1IrJBRJ7x3DshT0T+KiLfiMhKEYn2jDtORNaJyCYRea/yOn0RGSQiK0Rko2eagZ7Zd65274PXPGcDIiLzRSTJM5+/uPTS2w0rCKZBRGQ4cC0wVVXHAeXA94BOwDeqejqwCpjnmeRl4F5VHQNsrjb8NeBJVR2Lcy1Aumf4eOBOnHtTDACmikgkcAUw0jOf3/v2VRorCKahzgMmAF+LyAZPPADnUuk3PeO8CpwpIhFAF1Vd5Rn+EnC2iIQBvVX1PQBVLVLVAs8461U1TVUrcE7hjgNygCLgORG5Eqgc1/iIFQTTUAK8pKrjPI+hqvrbGsar61z4mi7JrVRc7e9yoIOqlgGTcK4GvRxY2sicTSNZQTANtRL4rojEgPdefv1wvkPf9YxzA/C5qmYDx0XkLM/w7wOr1LnXQ5qIXO6ZR7CIhNa2QM99IiJUdQnO5sQ4X7wwU6WD2wmY1kFVk0Tk18ByEQnAueLuZzg3LhkpIolANs5+BnAuxX3a84NPAW70DP8+8IyIPOyZx9V1LDYMeF9EQnDWLu5q5pdlTmJXO5pTIiJ5qtrZ7TxM87BNBmOMl60hGGO8bA3BGONlBcEY42UFwRjjZQXBGONlBcEY42UFwRjj9f9pnRRoWn+IEQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 norm of w: 1.3868197\n"
     ]
    }
   ],
   "source": [
    "fit_and_plot_tf2(0, lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAC1CAYAAACQyynAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAZQklEQVR4nO2de3SU5Z3HP7+5ZHLlloALxDaIN7yiRFdWt/VSLGitWl1rLd3Wekp7tj3a7tYKp2u79uzZsqvbqq1o0VVrUawrtkJFpCCIW7kYNGi4yUWEAHIJEEhCLjPz2z/eN5lJmsTJZeYN8/4+58yZ9/fc3u8k830vz/s8z4iqYhiG/wh4LcAwDG8w8xuGTzHzG4ZPMfMbhk8x8xuGTzHzG4ZPCXktIB2UlJRoWVmZ1zIMw3PWrl17UFWHd5aXleYvKyujoqLCaxmG4Tki8lFXeXbZbxg+ZcCf+UWkAJgFNAPLVfVZjyUZRlbgyZlfRJ4Ukf0iUtUhfbKIbBaRrSIy3U3+EvCiqn4L+GLGxRpGluLVZf/TwOTkBBEJAo8AU4CzgK+IyFlAKbDLLRbrl71Hm6DhUL80ZRgnKp6YX1VXAB3ddzGwVVW3q2oz8DxwPVCNcwCA/tAbj8HsK+CVf+5zU4ZxIjOQ7vlHkzjDg2P6vwUeBn4tItcCC7qqLCLTgGkAn/rUp7reSyAIfzsNBpd2XcYwfMBAMr90kqaqWg/c/kmVVXU2MBugvLy8+3nKE77RC3mGkV0MpEd91cDJSXEpsCdte4tF4S8Pw/o/pm0XhjGQGUjmfxs4TUTGiEgOcCswvycNiMh1IjK7trY2lcKw/iXYtrRXYg3jRMerR31zgZXAGSJSLSJ3qGoU+B7wGrAReEFV1/ekXVVdoKrTBg8e/MmFA0H4x5fhi7/q+QcwjCzAk3t+Vf1KF+kLgYUZE5LrHiSO7YMjO+HkizK2a8PwmoF02e8df5gGL34TYi1eKzGMjDGQevu9Y8r9oDEIhr1WYhgZI6vO/D3q8Etm+OkwYpyzvXUpRJv7X5xhDDCyyvw96vDrjAObYc5NsOqR/hVmGAOQrDJ/nxl+Bnz5d3Dxt5248ai3eoz+o6kOmhsS8ZFdUF+TiPeug9rdzrYqbFsGNducOB6Dqpdg/yYnjjZDxZOw9z0nbjkOb/4Cdr/jxI1H4c8/hV1vO3HDIVjwfdi52omPfez0MX200okPfwRzbk7EB7fA/1ydiD9+H2ZNhJ2rnHjX2/DLc5x2+oCZvyPjroOcfIjH4ZnrYeHdXisaGDQ3tJ8MdXRvwhzgGKM6aQGVnath65JEvHmRY6BWKp+Dtb9NxG/9GlbOSsRL/g1W3J+IX/4eLP1ZIn7uVnjtx4n48avglX9JxA+eC/PvTMQPnQd/vjcRz5oIbz6QiJ+YBGt+42yLwO9uhHXPO7EqvHg7bHRHl8db4E8/gG2vO3G0EZbeB7tWJ+JVs2D/+kS86RWodUevx5qdg83xw277cWg46JQDkACEcp13gFAeFI+FcJ4T5w2BMZ+BYA59QlWz5gVcB8w+9dRTtc9EW1T/7yHVd59z4pZG1Ze+o1q91oljUadMb4nHnTZjMSduqlOt2a7a0uTEx/ar7viLavNxJ67Zrvr+vES89z3VlbMS8Y63VBf/JFF/8yJHbyzqxJXPq/7uS4n9r3pMdfaViXjZz1UfvjARv3K36n+NTcR//K7qA2cm4nnTVH95biJ+4RuqD09IxHNvU31kYiKec7PqY59JxL/9ouoTVyfiZ7/s1Elub/6d7fe/5L5EvPAe1b/8KhEv+Znq2mcS8Zu/VN0wPxGveVx127JEvO4F1V0ViXjzItV9GxPxjrdUD+90tuNx1X0bVOsOJuLaPc7/rDVubkj8rQcQQIV25ZeuMk7k14QJSV/C/mLfRtX/PEV1wwIn3rNO9aeDVDe96sS731X9VbnqztVO/NEqx0y733HirUtVZ35adU+lE6//o1P/4yonfu9/nXj/Zid+91knrtnuxBVPOXHtbidePduJ6w448VuPqP6sRPX4kUT+L85xDjCt9WdfkTjYvPtce7O9P0/1lR8m4k0LVd/4r0S8bVl7c+16O/HZVR1zJJvp8M6EdlXVhsMJbaqOYYy0Y+bvL+LxxNH98E7V1/9D9eBWJ963UfX3/5gw88dVqv97e8LM+zY65jq0w4kPfKD6xv2qx/Y58aEPHcM3HHbi2j2qW19Xbap34voap41osxM3N6g2HEqY2TA6oTvzi5OfXZSXl6st4GkYICJrVbW8szzr8DMMn2LmNwyfklXm7/UIP8PwIVllfu3rCD/D8BFZZX7DMFLHzG8YPsXMbxg+xcxvGD4lq8xvvf2GkTpZZX7r7TeM1Mkq8xuGkTpmfsPwKWZ+w/ApZn7D8ClmfsPwKWZ+w/ApWWV+e85vGKmTVea35/yGkTpZZX7DMFLHzG8YPsXMbxg+xXfmP9rYwp837GP/0UavpRiGp/jO/LsPH+dbz1Twzs7DXksxDE/xnfkLIyEAjjVGPVZiGN7iO/MXuOavbzLzG/7Gh+YPAlDfHPNYiWF4i+/MHwkFyQkG7LLf8D1ZZf5Uh/cW5obsst/wPSGvBfQnqroAWFBeXv6t7soVRILUmfl9QUtLC9XV1TQ2Zvej3dzcXEpLSwmHwynXySrzp0phJGzm9wnV1dUUFRVRVlaGiHgtJy2oKjU1NVRXVzNmzJiU62XVZX+qFEaC1Nk9vy9obGykuLg4a40PICIUFxf3+OrGp+YPUd9s5vcL2Wz8VnrzGX1p/oJIyM78hu/xpfkLIyG75zcywpEjR5g1a1aP611zzTUcOXIkDYoSmPkNI410Zf5YrPtBZgsXLmTIkCHpkgX4tLe/IBKioTlGLK4EA9l/P2h4x/Tp09m2bRvjx48nHA5TWFjIyJEjqaysZMOGDdxwww3s2rWLxsZG7rrrLqZNmwZAWVkZFRUV1NXVMWXKFC677DLeeustRo8ezcsvv0xeXl6ftfnS/EW57vj+5iiDclN/Lmqc2Ny3YD0b9hzt1zbPGjWIn153dpf5M2fOpKqqisrKSpYvX861115LVVVV2yO5J598kmHDhnH8+HEuuugibrrpJoqLi9u1sWXLFubOncvjjz/OLbfcwrx585g6dWqftfvyst8m9xhecfHFF7d7Fv/www9z/vnnc8kll7Br1y62bNnyV3XGjBnD+PHjAZgwYQI7duzoFy0pnflF5C7gKeAY8ARwATBdVRf3i4oMU2jm9yXdnaEzRUFBQdv28uXLWbJkCStXriQ/P5/LL7+802f1kUikbTsYDHL8+PF+0ZLqmf+bqnoUuBoYDtwOzOwXBR5gc/qNTFFUVMSxY8c6zautrWXo0KHk5+ezadMmVq1alVFtqd7zt/aKXQM8parr5AQeOVHYes/fZNN6jfRSXFzMpZdeyjnnnENeXh4nnXRSW97kyZN57LHHOO+88zjjjDO45JJLMqotVfOvFZHFwBhghogUAfH0yeodInIdcN2pp57abbmCHOdj1zW1ZECV4Xeee+65TtMjkQivvvpqp3mt9/UlJSVUVVW1pf/whz/sN12pXvbfAUwHLlLVBiCMc+k/oEj1RztaL/vr7Mxv+JhUzT8R2KyqR0RkKvCvwAn7m1itl/11jXbmN/xLquZ/FGgQkfOBHwEfAc+kTVWasaW8DCN180dVVYHrgYdU9SGgKH2y0ost5WUYqXf4HRORGcDXgL8XkSDOff8JS0EkaM/5DV+T6pn/y0ATzvP+j4HRwP1pU5UBCnNtco/hb1Iyv2v4Z4HBIvIFoFFVT9h7fnAe95n5jXTT2ym9AA8++CANDQ39rChBSuYXkVuANcA/ALcAq0Xk5rSpygBFtoKvkQEGsvlTvef/Mc4z/v0AIjIcWAK8mC5h6aYgEuJQfbPXMowsJ3lK76RJkxgxYgQvvPACTU1N3Hjjjdx3333U19dzyy23UF1dTSwW495772Xfvn3s2bOHK664gpKSEpYtW9bv2lK95w+0Gt+lpgd1ByQFtqCHP3nqWnj3WWc71uLE637vxM0NTlw1z4kba514w3wnrq9x4s3uqLxj+z5xdzNnzmTs2LFUVlYyadIktmzZwpo1a6isrGTt2rWsWLGCRYsWMWrUKNatW0dVVRWTJ0/mzjvvZNSoUSxbtiwtxofUDbxIRF4TkW+IyDeAV4CFaVGUIYpsHT8jwyxevJjFixdzwQUXcOGFF7Jp0ya2bNnCueeey5IlS7jnnnt48803+aQRqv1FSpf9qnq3iNwEXIozyWe2qv4hrcrSTEHE7vl9ye2vJLaD4fZxTn77OHdw+7iguH1clJikkwqqyowZM/j2t7/9V3lr165l4cKFzJgxg6uvvpqf/OQnPWq7N6S8ko+qzgPmpVFLRnGW744RjysBW8rLSBPJU3o///nPc++99/LVr36VwsJCdu/eTTgcJhqNMmzYMKZOnUphYSFPP/10u7olJSVp0dat+UXkGKCdZQGqqoPSoioDtC3o0RylyJbyMtJE8pTeKVOmcNtttzFx4kQACgsLmTNnDlu3buXuu+8mEAgQDod59NFHAZg2bRpTpkxh5MiRabnvF2fUbnZRXl6uFRUV3ZaZu2YnM156n7emX8moIX1fDNEYmGzcuJFx48Z5LSMjdPZZRWStqpZ3Vv6E7rHvC8MKcgDscZ/hW3xr/mLX/DVmfsOn+Nf8hc6iiIfqmzxWYqSbbLy17UhvPqOPze+e+evszJ/N5ObmUlNTk9UHgNaf6M7Nze1RPV/+aAc4g3zCQeGgmT+rKS0tpbq6mgMHDngtJa3k5uZSWlraozoD3vwicgrO3ILBqtpvk4lEhOKCCDV1dtmfzYTD4XY/kmEkSOtlv4g8KSL7RaSqQ/pkEdksIltFZHp3bajqdlW9Ix36igtzrLff8C3pPvM/DfyapPX+3FWAHgEmAdXA2yIyHwgCP+9Q/5sdJhT1K8MKcjho5jd8SlrNr6orRKSsQ/LFwFZV3Q4gIs8D16vqz4EvpFNPR0oKI3x4sD6TuzSMAYMXvf2jgV1JcbWb1ikiUiwijwEXuOsIdlVumohUiEhFqp07xQV22W/4Fy86/DqbRdPlcxhVrQG+80mNqupsYDY4w3tTETKsMIeG5hgNzVHycwZ836dh9CtenPmrgZOT4lJgjwc6KClwBvrYs37Dj3hh/reB00RkjIjkALcC8z3QYeP7DV+T7kd9c4GVwBkiUi0id6hqFPge8BqwEXhBVdf30/6uE5HZtbWp/ZJY2yg/G+Jr+JB09/Z/pYv0haRhGTBVXQAsKC8v/1Yq5Uvc8f02ys/wI74d2w922W/4G1+bPz8nSG44YEN8DV+SVebv6T1/Yny/nfkN/5FV5lfVBao6rSdLHxcX2hBfw59klfl7gzPKzy77Df9h5i+McPCYnfkN/+F785cOzWPfsUYaW2JeSzGMjJJV5u9phx/AmJICVOGjmvT9GqphDESyyvy96fA7paQQgA8P1qVLlmEMSLLK/L2hrCQfgO02r9/wGb43f1FumOFFET48YOY3/IXvzQ/Ofb+t6GP4jawyf286/ABOMfMbPiSrzN+bDj9wzvw19c3UHm9JkzLDGHhklfl7y5iSAgB22Nnf8BFmfuCU4Y757dLf8BNmfuDkYfkExB73Gf7CzA9EQkFKh+az7YAN9DH8g5nf5fyTh7B6+yHi8ez9NVfDSCarzN/bR30Anz19OAfrmtj48dE0KDOMgUdWmb+3j/oAPnNaCQArPjjY37IMY0CSVebvCyMG5XLm3xSx4oPs/h13w2jFzJ/EZ08fTsVHh6hvinotxTDSjpk/ic+ePpyWmLJyW43XUgwj7Zj5k5hQNpTBeWHmrP7IaymGkXbM/ElEQkH+6fKxLN98gFXb7exvZDdm/g58/e/KGDk4l5mvbkLVnvkb2UtWmb8vz/lbyQ0H+cHnTqdy1xEeWrqlH9UZxsAiq8zfl+f8ydw8oZSbLizlwSVbmPnqJlpi8X5SaBgDh7T+Su+JSiAg3H/zeUTCAR57YxuvVu3lO58dy+fGncTwoojX8gyjX5BsvK8tLy/XioqKPrejqry+aT8PLP6AjXudYb9jhxdw5t8M4uRh+YwaksvwwghDC3IYnBdmUF6YotwQhTkhAgHp8/4No6+IyFpVLe8sz8783SAiXDXuJK48cwQb9h5l+eYDrNt1hPV7anlt/cdEu5kElBcOUhAJURAJkp8TIj8n2PbKCwfJywmRF3bjnCC5YSc9Nxxw34NEwgFyw0FyQ056bmt6KEAkFCAUzKq7NiPDmPlTQEQ4e9Rgzh6V6EuIxZWauiZq6ps55C4BdvR4C3VNUY41RqlvilLfHKO+KUpDc4yGZid9/9EmGlqiHG+Oc7w5yvGWGL2dSBgKiHMgCAfJCQaIhANt75GQk5YTSnoFA+3Swq3bQSEcdOJwKEAkGCAcEkKB1jLOdiipXCjgbIeCQtjNa90OJqcFBBG7ChqImPl7STAgjBiUy4hBuX1qR1VpjsVpbIlzvDnG8ZYYjW2vOI3RGE2t2256UzTuvmI0uWWa3bTmpLzmaJyGhijNMaUpGqMl5uS3vlpizr7TTTDgHARCAXG23YNHKCAEg0JQ3PRAwM0XApIo3+4lie1AUtxaPhAQAkJbWmtZEdqVDbrlWtsIiFOmLa81X5LfhUAAt2z7fGnNF9q11brdGgu0tQ1/XV4EhPZxa/tC8juEgwFGDcnr9f/FzO8xIkIkFCQSCjI4L5zx/asqLTElGncPCu4BIuoeGFpiznZLzDlYtMTiROPOdtSt52zHaYk779GYElM3zS0TjSuxmBKNu3FMicVbYyXupsfiuO/a9mqOOvXjqu3SY+rUc95pqx/XRNm2fKVtO1u6uT41LJ8VP7qi1/XN/D5HRMgJCTkEyM/xWk1m0NYDg7Y/UCjuAaKzPDdWxT2AJLbj7gEHcOs69VUTbbZ7T85XUJw21K0Prftz8zqUVQVVKIgE+/R3MPMbvkPEubXwO1nVXdwfI/wMwy9klfn7a4SfYfiBrDK/YRipY+Y3DJ+SlcN7ReQA0NWKHCXAQFql0/R0zUDSAiemnk+r6vDOMrLS/N0hIhVdjXX2AtPTNQNJC2SfHrvsNwyfYuY3DJ/iR/PP9lpAB0xP1wwkLZBlenx3z28YhoMfz/yGYeAj84vIZBHZLCJbRWS6B/s/WUSWichGEVkvIne56cNE5M8issV9H5phXUEReVdE/uS1HhEZIiIvisgm9+800Ss9IvID9/9UJSJzRSQ301pE5EkR2S8iVUlpXWoQkRnu93uziHz+k9r3hflFJAg8AkwBzgK+IiJnZVhGFPgXVR0HXAJ819UwHViqqqcBS904k9wFbEyKvdTzELBIVc8Eznd1ZVyPiIwG7gTKVfUcIAjc6oGWp4HJHdI61eB+l24FznbrzHK/912jbdMTs/cFTAReS4pnADM81vQyMAnYDIx000YCmzOoodT9Al0J/MlN80QPMAj4ELcfKik943qA0cAuYBjOzNc/AVd7pKUMqPqkv0fH7zTwGjCxu7Z9ceYn8c9spdpN8wQRKQMuAFYDJ6nqXgD3fUQGpTwI/AhIXs7HKz2nAAeAp9zbkCdEpMALPaq6G3gA2AnsBWpVdbEXWjqhKw09/o77xfydTd725DGHiBQC84Dvq+pRLzS4Or4A7FfVtV5p6EAIuBB4VFUvAOrJ/C0QAO599PXAGGAUUCAiU73Q0gN6/B33i/mrgZOT4lJgT6ZFiEgYx/jPqupLbvI+ERnp5o8E9mdIzqXAF0VkB/A8cKWIzPFQTzVQraqr3fhFnIOBF3o+B3yoqgdUtQV4Cfg7j7R0pCsNPf6O+8X8bwOnicgYEcnB6RiZn0kB4ixh+z/ARlX9RVLWfODr7vbXcfoC0o6qzlDVUlUtw/l7vK6qUz3U8zGwS0TOcJOuAjZ4pGcncImI5Lv/t6twOh89+dt0oCsN84FbRSQiImOA04A13baUic6cgfACrgE+ALYBP/Zg/5fhXIa9B1S6r2uAYpxOty3u+zAPtF1OosPPMz3AeKDC/Rv9ERjqlR7gPmATUAX8DohkWgswF6fPoQXnzH5HdxqAH7vf783AlE9q30b4GYZP8ctlv2EYHTDzG4ZPMfMbhk8x8xuGTzHzG4ZPMfMbGUVELm+dQWh4i5nfMHyKmd/oFBGZKiJrRKRSRH7jzvuvE5H/FpF3RGSpiAx3y44XkVUi8p6I/KF1jrmInCoiS0RknVtnrNt8YdK8/WfdUXSIyEwR2eC284BHH903mPmNv0JExgFfBi5V1fFADPgqUAC8o6oXAm8AP3WrPAPco6rnAe8npT8LPKKq5+OMjd/rpl8AfB9nbYVTgEtFZBhwI3C2286/p/dTGmZ+ozOuAiYAb4tIpRufgjP19/dumTnAZSIyGBiiqm+46b8FPiMiRcBoVf0DgKo2qmqDW2aNqlarahxnmHMZcBRoBJ4QkS8BrWWNNGHmNzpDgN+q6nj3dYaq/lsn5bobG97db2A3JW3HgJCqRoGLcWY93gAs6qFmo4eY+Y3OWArcLCIjoG3duE/jfF9udsvcBvyfqtYCh0Xk7930rwFvqLNWQbWI3OC2ERGR/K526K5zMFhVF+LcEoxPxwczEoS8FmAMPFR1g4j8K7BYRAI4s8q+i7PAxtkishaoxekXAGdq6WOuubcDt7vpXwN+IyI/c9v4h252WwS8LCK5OFcNP+jnj2V0wGb1GSkjInWqWui1DqN/sMt+w/ApduY3DJ9iZ37D8ClmfsPwKWZ+w/ApZn7D8ClmfsPwKWZ+w/Ap/w+P4JDLFWkOuQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 norm of w: 0.3116793\n"
     ]
    }
   ],
   "source": [
    "fit_and_plot_tf2(3, lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "3.12_weight-decay.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3.7.4 64-bit ('tf': conda)",
   "language": "python",
   "name": "python37464bittfcondac1d40346f2a44515bd7c51c557ecf43c"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}